
import pandas as pd
import numpy as np
from tqdm.notebook import tqdm

from bs4 import BeautifulSoup
import re
from pymorphy2 import MorphAnalyzer
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GroupKFold
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from sklearn.metrics import f1_score
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import pairwise_distances

from sklearn.linear_model import LogisticRegressionCV, LogisticRegression
import xgboost as xgb
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
import lightgbm as lgb


nltk.download('stopwords')
nltk.download('wordnet')

stopwords_ru = stopwords.words("russian")
stopwords_en = stopwords.words("english")
patterns = '[^A-Za-zА-Яа-я]'
morph = MorphAnalyzer()
lemmatizer = WordNetLemmatizer()


def is_russian(word):
    for ch in word:
        if ord('а') <= ord(ch) <= ord('я'):
            return True
    return False


def lemmatize(text):
    text = re.sub(patterns, ' ', text)
    tokens = []
    for token in text.split():
        token = token.strip().lower()
        is_rus = is_russian(token)
        if len(token) > 1 and is_rus and token not in stopwords_ru:
            token = morph.normal_forms(token)[0]
            tokens.append(token)
        elif not is_rus and len(token) > 1 and token not in stopwords_en:
            token = lemmatizer.lemmatize(token)
            tokens.append(token)
    return tokens


parsed_data = {}
with open('data/docs_titles.tsv', 'r') as docs_titles:
    for num_line, line in tqdm(enumerate(docs_titles)):
        if num_line == 0:
            continue
        data = line.strip().split('\t', 1)
        doc_id = int(data[0])
        if len(data) == 1:
            title_tokens = ['']
        else:
            title_tokens = lemmatize(data[1])
        with open('data/content/' + str(doc_id) + '.dat', 'r') as content:
            content.readline()
            soup = BeautifulSoup(content.read(), "lxml")
            for script in soup(['script', 'style']):
                script.extract()
            text = soup.get_text(separator=' ')
            content_tokens = lemmatize(text)
        parsed_data[doc_id] = (' '.join(title_tokens), ' '.join(content_tokens))
doc_count = num_line
print(doc_count)


df_parsed_data = pd.DataFrame.from_dict(parsed_data, orient='index', columns=['title', 'text'])
df_parsed_data.head()


df_parsed_data.shape


df_parsed_data.to_csv('parsed_data_1.csv', index=True, index_label='doc_id')


def vector_process(x, n):
    h = len(x) / n
    ret = np.zeros(n)
    for j in range(n):
        ret[j] = np.mean(x[int(np.floor(j * h)): int(np.ceil((j + 1) * h))])
    return ret


N_title = 20
N_content = 25
# N_features = 40
stop_words = ['ru', 'http', 'https', 'com', 'www']
ngram_range = (1, 2)
max_df=0.95
min_df=3


def extract_features(df_groups, df_data, return_target=False):
    groups_data = {}
    for i in tqdm(range(len(df_groups))):
        new_doc = df_groups.iloc[i]
        group_id = new_doc['group_id']
        doc_id = new_doc['doc_id']
        if return_target:
            target = new_doc['target']
        doc_data = df_data[df_data.doc_id == doc_id]
        doc_data = doc_data.iloc[0]
        title = str(doc_data['title'])
        content = str(doc_data['text'])
        if group_id not in groups_data:
            groups_data[group_id] = []
        if return_target:
            groups_data[group_id].append((doc_id, title, content, target))
        else:
            groups_data[group_id].append((doc_id, title, content))
    
    y = []
    groups = []

# top title/content length intersections,
# length title/content,
# abs(mean_by_group(len_content_intersect) - len_content_intersect)
# abs(mean_by_group(len_title_intersect) - len_title_intersect)
    X_1 = []
    for new_group in tqdm(groups_data):
        mean_content_intersect = []
        mean_title_intersect = []
        docs = groups_data[new_group]
        for k, doc in enumerate(docs):
            groups.append(new_group)
            if return_target:
                y.append(doc[-1])
            title = doc[1]
            content = doc[2]
            title_intersections = []
            content_intersections = []
            title_words = set(title.strip().split())
            len_title = len(title_words)
            content_words = set(content.strip().split())
            len_content = len(content_words)
            for j in range(0, len(docs)):
                if k == j:
                    continue
                title_j = docs[j][1]
                content_j = docs[j][2]
                
                title_words_j = set(title_j.strip().split())
                content_words_j = set(content_j.strip().split())
                
                title_intersect_len = len(title_words.intersection(title_words_j))
                title_union_len = len(title_words_j) + len_title - title_intersect_len
                title_intersections.append(title_intersect_len / title_union_len)
                
                content_intersect_len = len(content_words.intersection(content_words_j))
                content_union_len = len(content_words_j) + len_content - content_intersect_len
                content_intersections.append(content_intersect_len / content_union_len)

            title_features = np.zeros(N_title)
            tmp = list(sorted(title_intersections, reverse=True)[0:N_title])
            title_features[:len(tmp)] = tmp

            content_features = np.zeros(N_content)
            tmp = list(sorted(content_intersections, reverse=True)[0:N_content])
            content_features[:len(tmp)] = tmp

            #content_features = np.array(vector_process(list(sorted(content_intersections, reverse=True)), N_features))

            len_features = np.zeros(2)
            len_features[0] = len(title.strip().split())
            len_features[1] = np.log(len(content.strip().split()))

            mean_content_intersection_feature = np.array([np.mean(content_intersections)])
            mean_title_intersection_feature = np.array([np.mean(title_intersections)])
            
            mean_content_intersect.append(mean_content_intersection_feature[0])
            mean_title_intersect.append(mean_title_intersection_feature[0])
            
            group_size_feature = np.array([len(docs)])

            X_1.append(np.concatenate((title_features,
                                       content_features,
                                       len_features,
                                       group_size_feature,
                                       mean_content_intersection_feature,
                                       mean_title_intersection_feature), axis=0))

        mean_content_intersect_by_group = np.mean(mean_content_intersect)
        mean_title_intersect_by_group = np.mean(mean_title_intersect)
        for x in X_1[-len(docs):]:
            x[-2] = np.abs(mean_content_intersect_by_group - x[-2])
            x[-1] = np.abs(mean_title_intersect_by_group - x[-1])

    X_1 = np.array(X_1)
    y = np.array(y)
    groups = np.array(groups)

# Tfidf features
    X_2 = []
    for group_id in tqdm(groups_data):
        corpus = []
        for doc in groups_data[group_id]:
            corpus.append(doc[2])
        vectorizer = TfidfVectorizer(stop_words=stop_words, ngram_range=ngram_range, max_df=max_df, min_df=min_df)
        tfidf_vectors = vectorizer.fit_transform(corpus)
        pairwise_dist = pairwise_distances(tfidf_vectors, metric='cosine', n_jobs=-1)
        for dist_vector in pairwise_dist:
            dist_vector = np.sort(dist_vector)
            X_2.append(dist_vector[1:26])

    X_2 = np.asarray(X_2)
    X = np.concatenate((X_1, X_2), axis=1)
    if return_target:
        return (groups, X, y)
    return (groups, X)


df_train_groups = pd.read_csv('data/train_groups.csv')

# lemmatize tokens
df_parsed_data = pd.read_csv('parsed_data_1.csv')

# stemmatize tokens
# df_parsed_data = pd.read_csv('raw_texts.csv')

df_parsed_data.tail()


groups_train, X_train, y_train = extract_features(df_train_groups, df_parsed_data, return_target=True)
print(groups_train.shape, X_train.shape, y_train.shape)

(11690,) (11690, 75) (11690,)


X_train_src = np.array(X_train)


X_train = np.array(X_train_src)
X_train.shape

(11690, 75)


scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)


df_test_groups = pd.read_csv('data/test_groups.csv')


groups_test, X_test = extract_features(df_test_groups, df_parsed_data, return_target=False)
print(groups_test.shape, X_test.shape)

(16627,) (16627, 75)


X_test_src = np.array(X_test)


X_test = scaler.transform(X_test)


n_splits = 5
cv = GroupKFold(n_splits=n_splits)
cv_gen = cv.split(X_train, y_train, groups_train)
clf_logreg = LogisticRegressionCV(cv=cv_gen, scoring='f1', n_jobs=-1, max_iter=500, Cs=100)
clf_logreg.fit(X_train, y_train)
mean_scores = np.mean(clf_logreg.scores_[1], axis=0)
mean_scores

array([0.57251868, 0.58033796, 0.59073517, 0.60010451, 0.60622284,
       0.61346431, 0.61942043, 0.62346059, 0.62722561, 0.62842465,
       0.63084637, 0.63330728, 0.63653556, 0.63717716, 0.63882454,
       0.64095818, 0.64268425, 0.64333992, 0.6465891 , 0.64861666,
       0.64987049, 0.65051561, 0.65234122, 0.65513552, 0.65638915,
       0.65666575, 0.65676318, 0.65585781, 0.6572287 , 0.65784254,
       0.65773739, 0.65809033, 0.65813367, 0.65710569, 0.65722715,
       0.6590217 , 0.65837226, 0.65948567, 0.66033412, 0.66153731,
       0.66147893, 0.66204135, 0.66135648, 0.66128   , 0.66050625,
       0.66135283, 0.66190882, 0.66165002, 0.66160951, 0.66213237,
       0.66175979, 0.66149573, 0.6611123 , 0.66211917, 0.66237861,
       0.66173267, 0.66148062, 0.6609774 , 0.66052182, 0.66066   ,
       0.66077998, 0.66101737, 0.66077745, 0.6609249 , 0.66060933,
       0.66075678, 0.66070174, 0.66055411, 0.66055411, 0.66055411,
       0.66055411, 0.66103879, 0.66103879, 0.66103879, 0.66096288,
       0.66096288, 0.66096288, 0.66096288, 0.66096288, 0.66096288,
       0.66096288, 0.66111051, 0.66078316, 0.66063536, 0.66063536,
       0.66063536, 0.66063536, 0.66086782, 0.66063536, 0.66086782,
       0.66086782, 0.66086782, 0.66086782, 0.66086782, 0.66086782,
       0.66086782, 0.66086782, 0.66086782, 0.66086782, 0.66086782])


np.max(mean_scores, axis=-1)

0.662378608221785


y_pred = clf_logreg.predict(X_test)
y_pred.shape


np.unique(y_pred, return_counts=True)


submission = pd.DataFrame({'pair_id': df_test_groups.pair_id, 'target': y_pred.astype(np.int32)})
submission = submission.set_index(['pair_id'])
submission.head()


submission.to_csv('LogReg1.csv')


param = {'learning_rate': 0.3,
         'max_depth': 4,
         'max_leaves': 4,
         'subsample': 0.9,
         'colsample_bytree': 1,
         'reg_alpha': 50,
         'reg_lambda': 100,
         'booster': 'gbtree', #gbtree, dart
         'objective': 'binary:logistic',
         'eval_metric': 'logloss'}


cv = GroupKFold(n_splits=5)
scores = []
for train, val in cv.split(X_train, y_train, groups_train):
    dtrain = xgb.DMatrix(X_train[train], label=y_train[train])
    bst = xgb.train(param, dtrain, 40)
    dval = xgb.DMatrix(X_train[val])
    y_pred = bst.predict(dval)
    threshold = 0.38
    y_pred[y_pred >= threshold] = 1
    y_pred[y_pred < threshold] = 0
    y_pred = y_pred.astype(np.int)
    scores.append(f1_score(y_true=y_train[val], y_pred=y_pred))
scores

[0.6983240223463687,
 0.6269982238010657,
 0.7901938426453821,
 0.7563527653213752,
 0.8052631578947369]


np.mean(scores)

0.7354264024017857


threshold = 0.345
n_estimators = 26


n_splits = 5

def validate(params, X, y, groups, threshold=0.35):
    f_score = np.zeros(n_splits)
    kf = GroupKFold(n_splits)
    for i, (idxs_train, idxs_val) in enumerate(kf.split(X, y, groups)):
        dtrain = xgb.DMatrix(X_train[idxs_train], label=y_train[idxs_train])
        bst = xgb.train(params, dtrain, n_estimators)
        
        dval = xgb.DMatrix(X_train[idxs_val])
        y_pred = bst.predict(dval)
        
        y_pred[y_pred >= threshold] = 1
        y_pred[y_pred < threshold] = 0
        y_pred = y_pred.astype(np.int)
        f_score[i] = f1_score(y[idxs_val], y_pred)

    return f_score.mean()


fixed_params = {
    'objective': 'binary:logistic',
    'eval_metric': 'logloss',
    'booster': 'gbtree', #gbtree
}


grid_params = {
    'threshold': np.linspace(0.3, 0.6, 20),
    "learning_rate": np.linspace(0.0001, 1.0, num=50),
    ("max_depth", "max_leaves"): (range(3, 6), range(4, 40, 2)),
    ("subsample", "colsample_bytree"): (np.linspace(0.6, 1.0, num=10),np.linspace(0.6, 1.0, num=10)),
    ("reg_alpha", "reg_lambda"): ([0, 0.05, 0.1, 0.5, 1, 5, 10, 50, 90, 100, 120, 150, 500],
                                  [0, 0.05, 0.1, 0.5, 1, 5, 10, 50, 90, 100, 120, 150, 500]),
}


params = {
    "learning_rate": 0.1,
    "max_depth": 4,
    "max_leaves": 9,
    "subsample": 0.85,
    "colsample_bytree": 1.,
    "reg_alpha": 0.1,
    "reg_lambda": 100,
}


score = []
for alpha in tqdm(grid_params["learning_rate"]):
    pp = params.copy()
    pp.update(fixed_params)
    pp["learning_rate"] = alpha
    score.append(validate(pp, X_train, y_train, groups_train))
    print(score[-1])

params["learning_rate"] = grid_params["learning_rate"][np.argmax(score)]
max_score = np.max(score)
print(f"Best learning rate {params['learning_rate']} with score {max_score}")

0.689603623441699
0.696790435671935
0.7050398281010961
0.7053820925064547
0.7103773540299806
0.7111518434878996
0.7081635037928495
0.7138080757623069
0.7137227730598493
0.7131352908027926
0.7068083705397409
0.7096240595070001
0.7051841386539357
0.7067672580500564
0.7043189856156318
0.7006242976490199
0.7005419582021073
0.7023361071928083
0.6923485362761588
0.6982259859521033
0.6944924213069165
0.6946506420575529
0.6956736392067665
0.6944397527600238
0.6928458703612952
0.6829823932846609
0.6738149779852336
0.6810268066936257
0.686136616241269
0.6767427475187385
0.6806575954201336
0.6741774706246344
0.6736664467497142
0.6720453435097877
0.6784826579515058
0.6702200090630728
0.6720116742813124
0.6758272508378631
0.6667941737996721
0.6693054369591218
0.6724196719663318
0.6633754814248759
0.6615233817553776
0.6682706997905059
0.6567166434559295
0.6545484445430371
0.666830037825994
0.6561416514767932
0.6515138305837673
0.6473039629778603
Best learning rate 0.14294285714285712 with score 0.7138080757623069


max_depths, num_leavess = grid_params[("max_depth", "max_leaves")]
score = []
for max_depth in tqdm(max_depths):
    for num_leaves in num_leavess:
        if num_leaves > 2 ** max_depth:
            break
        pp = params.copy()
        pp.update(fixed_params)
        pp["max_depth"] = max_depth
        pp["max_leaves"] = num_leaves
        score.append((max_depth, num_leaves, validate(pp, X_train, y_train, groups_train)))
        print(score[-1])

best_max_depth, best_num_leaves, best_score = max(score, key=lambda x: x[2])
params['max_depth'] = best_max_depth
params['max_leaves'] = best_num_leaves
print(f"Best max_depth {params['max_depth']}, max_leaves {params['max_leaves']} with score {best_score}")

(3, 4, 0.7108904425712608)
(3, 5, 0.7108904425712608)
(3, 6, 0.7108904425712608)
(3, 7, 0.7108904425712608)
(3, 8, 0.7108904425712608)
(4, 4, 0.7138080757623069)
(4, 5, 0.7138080757623069)
(4, 6, 0.7138080757623069)
(4, 7, 0.7138080757623069)
(4, 8, 0.7138080757623069)
(4, 9, 0.7138080757623069)
(4, 10, 0.7138080757623069)
(4, 11, 0.7138080757623069)
(4, 12, 0.7138080757623069)
(4, 13, 0.7138080757623069)
(4, 14, 0.7138080757623069)
(4, 15, 0.7138080757623069)
(4, 16, 0.7138080757623069)
(5, 4, 0.7135966636806402)
(5, 5, 0.7135966636806402)
(5, 6, 0.7135966636806402)
(5, 7, 0.7135966636806402)
(5, 8, 0.7135966636806402)
(5, 9, 0.7135966636806402)
(5, 10, 0.7135966636806402)
(5, 11, 0.7135966636806402)
(5, 12, 0.7135966636806402)
(5, 13, 0.7135966636806402)
(5, 14, 0.7135966636806402)
(5, 15, 0.7135966636806402)
(5, 16, 0.7135966636806402)
(5, 17, 0.7135966636806402)
(5, 18, 0.7135966636806402)
(5, 19, 0.7135966636806402)
(5, 20, 0.7135966636806402)
(5, 21, 0.7135966636806402)
(5, 22, 0.7135966636806402)
(5, 23, 0.7135966636806402)
(5, 24, 0.7135966636806402)
(5, 25, 0.7135966636806402)
(5, 26, 0.7135966636806402)
(5, 27, 0.7135966636806402)
(5, 28, 0.7135966636806402)
(5, 29, 0.7135966636806402)
(5, 30, 0.7135966636806402)
(5, 31, 0.7135966636806402)
(5, 32, 0.7135966636806402)
Best max_depth 4, max_leaves 4 with score 0.7138080757623069


bagging_fractions, feature_fractions = grid_params[("subsample", "colsample_bytree")]
score = []
for bagging_fraction in tqdm(bagging_fractions):
    for feature_fraction in feature_fractions:
        pp = params.copy()
        pp.update(fixed_params)
        pp["subsample"] = bagging_fraction
        pp["colsample_bytree"] = feature_fraction
        score.append((bagging_fraction, feature_fraction, validate(pp, X_train, y_train, groups_train)))
        print(score[-1])


best_bagging_fraction, best_feature_fraction, best_score = max(score, key=lambda x: x[2])
params['subsample'] = best_bagging_fraction
params['colsample_bytree'] = best_feature_fraction
print(f"Best subsample {params['subsample']}, colsample_bytree {params['colsample_bytree']} with score {best_score}")

Best subsample 0.6666666666666666, colsample_bytree 0.6684210526315789 with score 0.7185779327896118


l1_lambdas, l2_lambdas = grid_params[("reg_alpha", "reg_lambda")]
score = []
for l1_lambda in tqdm(l1_lambdas):
    for l2_lambda in l2_lambdas:
        pp = params.copy()
        pp.update(fixed_params)
        pp["reg_alpha"] = l1_lambda
        pp["reg_lambda"] = l2_lambda
        score.append((l1_lambda, l2_lambda, validate(pp, X_train, y_train, groups_train)))
        print(score[-1])

best_l1_lambda, best_l2_lambda, best_score = max(score, key=lambda x: x[2])
params['reg_alpha'] = best_l1_lambda
params['reg_lambda'] = best_l2_lambda
print(f"Best reg_alpha {params['reg_alpha']}, reg_lambda {params['reg_lambda']} with score {best_score}")

(0, 0, 0.7185779327896118)
(0, 0.05, 0.7141773777187559)
(0, 0.1, 0.7113068225264103)
(0, 0.5, 0.7133331592098231)
(0, 1, 0.7154954259736056)
(0, 5, 0.7166267172618546)
(0, 10, 0.7168381736120447)
(0, 50, 0.7079088309512491)
(0, 100, 0.7083024125380882)
(0, 150, 0.7065276795014034)
(0, 500, 0.7008495042286773)
(0.05, 0, 0.7115872894443581)
(0.05, 0.05, 0.7158087686322941)
(0.05, 0.1, 0.713202362625901)
(0.05, 0.5, 0.7172849789255871)
(0.05, 1, 0.7132993505373622)
(0.05, 5, 0.7127116500971891)
(0.05, 10, 0.7196537735878901)
(0.05, 50, 0.7074649428314914)
(0.05, 100, 0.7099651699957936)
(0.05, 150, 0.7065276795014034)
(0.05, 500, 0.7008967181237914)
(0.1, 0, 0.7108134106818846)
(0.1, 0.05, 0.7123229808976447)
(0.1, 0.1, 0.7121569565553013)
(0.1, 0.5, 0.7182270565914046)
(0.1, 1, 0.7137171014218657)
(0.1, 5, 0.714504250523057)
(0.1, 10, 0.7155740923439379)
(0.1, 50, 0.7069564141941493)
(0.1, 100, 0.7099651699957936)
(0.1, 150, 0.7064104016534649)
(0.1, 500, 0.7008042585649533)
(0.5, 0, 0.7147993583170579)
(0.5, 0.05, 0.7159936217132948)
(0.5, 0.1, 0.7136334537417308)
(0.5, 0.5, 0.7121549974047927)
(0.5, 1, 0.7112201380687918)
(0.5, 5, 0.7124628047650856)
(0.5, 10, 0.7150350419919842)
(0.5, 50, 0.7077453194407436)
(0.5, 100, 0.7097294941181185)
(0.5, 150, 0.7056003475611237)
(0.5, 500, 0.69974665865845)
(1, 0, 0.7123906350612764)
(1, 0.05, 0.7080173779977555)
(1, 0.1, 0.712431056576627)
(1, 0.5, 0.7114730583720201)
(1, 1, 0.7126127858903256)
(1, 5, 0.7129423381765216)
(1, 10, 0.7166988885601528)
(1, 50, 0.7098927223454821)
(1, 100, 0.7094004626568037)
(1, 150, 0.7066955616195718)
(1, 500, 0.7011433850511632)
(5, 0, 0.7176382749954985)
(5, 0.05, 0.7188681025680099)
(5, 0.1, 0.7139959578560313)
(5, 0.5, 0.7120448070002572)
(5, 1, 0.7165981927353396)
(5, 5, 0.7154209297840559)
(5, 10, 0.7152109059445486)
(5, 50, 0.7128162150053194)
(5, 100, 0.7065473323883588)
(5, 150, 0.7085708226484209)
(5, 500, 0.6984058757283674)
(10, 0, 0.7153701542398798)
(10, 0.05, 0.7150873328362302)
(10, 0.1, 0.7147186964839335)
(10, 0.5, 0.7115827228719545)
(10, 1, 0.7106487662452354)
(10, 5, 0.709253041811996)
(10, 10, 0.7132807367131075)
(10, 50, 0.7095799739358536)
(10, 100, 0.7059595731849783)
(10, 150, 0.7084990780185765)
(10, 500, 0.7002799368815658)
(50, 0, 0.7072451345505874)
(50, 0.05, 0.7073670397784463)
(50, 0.1, 0.7073670397784463)
(50, 0.5, 0.7070218039423011)
(50, 1, 0.7074641457445008)
(50, 5, 0.707486982974156)
(50, 10, 0.7066297897559941)
(50, 50, 0.7030061910464692)
(50, 100, 0.7042144851183908)
(50, 150, 0.7026141728963701)
(50, 500, 0.6965008884243786)
(100, 0, 0.7024841677255054)
(100, 0.05, 0.7024841677255054)
(100, 0.1, 0.7023413058205847)
(100, 0.5, 0.7023413058205847)
(100, 1, 0.7028941426371283)
(100, 5, 0.7014310201293524)
(100, 10, 0.7020473058807881)
(100, 50, 0.7013518356675504)
(100, 100, 0.705116142337452)
(100, 150, 0.700423964561742)
(100, 500, 0.6959501097189397)
(150, 0, 0.6970018699019982)
(150, 0.05, 0.6970018699019982)
(150, 0.1, 0.6970018699019982)
(150, 0.5, 0.6980287645411267)
(150, 1, 0.6974885130636623)
(150, 5, 0.6964966659915124)
(150, 10, 0.6977130557732997)
(150, 50, 0.6994604042126276)
(150, 100, 0.6961063133020617)
(150, 150, 0.6967072397867011)
(150, 500, 0.6950273745616927)
(500, 0, 0.6410713947470444)
(500, 0.05, 0.6410713947470444)
(500, 0.1, 0.6410713947470444)
(500, 0.5, 0.6410713947470444)
(500, 1, 0.6410709352330953)
(500, 5, 0.6415867134118984)
(500, 10, 0.6414848396102327)
(500, 50, 0.6429843863570334)
(500, 100, 0.6427688881574402)
(500, 150, 0.646385451102014)
(500, 500, 0.6486457955771212)
Best reg_alpha 0.05, reg_lambda 10 with score 0.7196537735878901


thresholds = grid_params['threshold']
score = []
for threshold in tqdm(thresholds):
    pp = params.copy()
    pp.update(fixed_params)
    score.append(validate(pp, X_train, y_train, groups_train, threshold=threshold))
    print(score[-1])

best_threshold = grid_params["threshold"][np.argmax(score)]
max_score = np.max(score)
print(f"Best threshold {best_threshold} with score {max_score}")

0.7263819856167941
0.7288254902525245
0.7294781701225146
0.7291084102968577
0.7308562450975279
0.7322042964154474
0.7338227408269565
0.7337123094103889
0.7339829890720615
0.7348103440911253
0.7346168761832922
0.7339625522588452
0.7317715924423756
0.7295749876774452
0.7277961269288775
0.7274451820337456
0.7266088680013167
0.7247258247704491
0.7213710233944347
0.7196537735878901
Best threshold 0.39473684210526316 with score 0.7348103440911253


best_threshold = 0.39


params

{'learning_rate': 0.14294285714285712,
 'max_depth': 4,
 'max_leaves': 4,
 'subsample': 0.6666666666666666,
 'colsample_bytree': 0.6684210526315789,
 'reg_alpha': 0.05,
 'reg_lambda': 10}


param = {'learning_rate': 0.3,
         'max_depth': 4,
         'max_leaves': 4,
         'subsample': 0.9,
         'colsample_bytree': 1,
         'reg_alpha': 50,
         'reg_lambda': 100,
         'booster': 'gbtree', #gbtree, dart
         'objective': 'binary:logistic',
         'eval_metric': 'logloss'}
n_estimators = 40
threshold = 0.38


dtrain = xgb.DMatrix(X_train, label=y_train)
bst = xgb.train(param, dtrain, n_estimators)
dtest = xgb.DMatrix(X_test)
y_pred = bst.predict(dtest)
y_pred[y_pred >= threshold] = 1
y_pred[y_pred < threshold] = 0
y_pred = y_pred.astype(np.int)
y_pred.shape


np.unique(y_pred, return_counts=True)


submission = pd.DataFrame({'pair_id': df_test_groups.pair_id, 'target': y_pred.astype(np.int32)})
submission = submission.set_index(['pair_id'])
submission.head()


submission.to_csv('XGBoost3.csv')


n_splits = 5
def lgbm_f1_score(y_pred, data):
    y_true = data.get_label()
    y_pred = np.where(y_pred < 0.5, 0, 1)
    return "f1_score", f1_score(y_true, y_pred), True

def validate(params, X, y, groups, threshold=0.4):
    f_score = np.zeros(n_splits)
    kf = GroupKFold(n_splits)
    for i, (idxs_train, idxs_val) in enumerate(kf.split(X, y, groups)):
        train_set = lgb.Dataset(X[idxs_train], y[idxs_train])
        val_set = lgb.Dataset(X[idxs_val], y[idxs_val])
        
        model = lgb.train(params, train_set, verbose_eval=10,
                          valid_sets=[train_set, val_set], feval=lgbm_f1_score)

        y_pred = model.predict(X[idxs_val])
        y_pred = np.where(y_pred < threshold, 0, 1)
        f_score[i] = f1_score(y[idxs_val], y_pred)

    return f_score.mean()


fixed_params = {
     "objective": "binary",
    "metric": None,
    "is_unbalance": True,
    "boosting": "gbdt",
    "num_boost_round": 36,
    "early_stopping_rounds": 30,
    "bagging_freq": 1,
    "n_jobs": -1,
    "verbose": -1,
    "seed": 42,
}


grid_params = {
    'threshold': np.linspace(0.3, 0.5, 20),
    "learning_rate": np.linspace(0.0001, 1.0, num=50),
    ("max_depth", "num_leaves"): (range(3, 6), range(4, 60)),
    ("bagging_fraction", "feature_fraction"): (np.linspace(0.5, 1.0, num=10),np.linspace(0.1, 1.0, num=20)),
    ("lambda_l1", "lambda_l2"): ([0, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100, 150, 500],
                                 [0, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100, 150, 500]),
}

params = {
    "learning_rate": 0.1,
    "max_depth": 4,
    "num_leaves": 9,
    "bagging_fraction": 0.85,
    "feature_fraction": 1.,
    "lambda_l1": 0,
    "lambda_l2": 0,
}


score = []
for alpha in grid_params["learning_rate"]:
    pp = params.copy()
    pp.update(fixed_params)
    pp["learning_rate"] = alpha
    score.append(validate(pp, X_train, y_train, groups_train))

params["learning_rate"] = grid_params["learning_rate"][np.argmax(score)]
max_score = np.max(score)
print(f"Best learning rate {params['learning_rate']} with score {max_score}")

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.617449	training's f1_score: 0	valid_1's binary_logloss: 0.53277	valid_1's f1_score: 0
[20]	training's binary_logloss: 0.616959	training's f1_score: 0	valid_1's binary_logloss: 0.532433	valid_1's f1_score: 0
[30]	training's binary_logloss: 0.61647	training's f1_score: 0	valid_1's binary_logloss: 0.532101	valid_1's f1_score: 0
Early stopping, best iteration is:
[1]	training's binary_logloss: 0.617891	training's f1_score: 0	valid_1's binary_logloss: 0.533073	valid_1's f1_score: 0
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.615144	training's f1_score: 0	valid_1's binary_logloss: 0.540683	valid_1's f1_score: 0
[20]	training's binary_logloss: 0.614619	training's f1_score: 0	valid_1's binary_logloss: 0.540383	valid_1's f1_score: 0

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[30]	training's binary_logloss: 0.614094	training's f1_score: 0	valid_1's binary_logloss: 0.540085	valid_1's f1_score: 0
Early stopping, best iteration is:
[1]	training's binary_logloss: 0.615619	training's f1_score: 0	valid_1's binary_logloss: 0.540953	valid_1's f1_score: 0
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.581634	training's f1_score: 0	valid_1's binary_logloss: 0.675655	valid_1's f1_score: 0
[20]	training's binary_logloss: 0.581131	training's f1_score: 0	valid_1's binary_logloss: 0.675012	valid_1's f1_score: 0

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[30]	training's binary_logloss: 0.58063	training's f1_score: 0	valid_1's binary_logloss: 0.674373	valid_1's f1_score: 0
Early stopping, best iteration is:
[1]	training's binary_logloss: 0.582089	training's f1_score: 0	valid_1's binary_logloss: 0.676235	valid_1's f1_score: 0
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.604449	training's f1_score: 0	valid_1's binary_logloss: 0.579785	valid_1's f1_score: 0
[20]	training's binary_logloss: 0.603956	training's f1_score: 0	valid_1's binary_logloss: 0.579305	valid_1's f1_score: 0

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[30]	training's binary_logloss: 0.603462	training's f1_score: 0	valid_1's binary_logloss: 0.578828	valid_1's f1_score: 0
Early stopping, best iteration is:
[1]	training's binary_logloss: 0.604897	training's f1_score: 0	valid_1's binary_logloss: 0.580217	valid_1's f1_score: 0

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.573876	training's f1_score: 0	valid_1's binary_logloss: 0.709745	valid_1's f1_score: 0
[20]	training's binary_logloss: 0.573383	training's f1_score: 0	valid_1's binary_logloss: 0.708991	valid_1's f1_score: 0
[30]	training's binary_logloss: 0.572892	training's f1_score: 0	valid_1's binary_logloss: 0.708239	valid_1's f1_score: 0
Early stopping, best iteration is:
[1]	training's binary_logloss: 0.574321	training's f1_score: 0	valid_1's binary_logloss: 0.71043	valid_1's f1_score: 0
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.537794	training's f1_score: 0	valid_1's binary_logloss: 0.476708	valid_1's f1_score: 0
[20]	training's binary_logloss: 0.486324	training's f1_score: 0.548936	valid_1's binary_logloss: 0.439828	valid_1's f1_score: 0.443058


max_depths, num_leavess = grid_params[("max_depth", "num_leaves")]
score = []
for max_depth in max_depths:
    for num_leaves in num_leavess:
        if num_leaves > 2 ** max_depth:
            break
        pp = params.copy()
        pp.update(fixed_params)
        pp["max_depth"] = max_depth
        pp["num_leaves"] = num_leaves
        score.append((max_depth, num_leaves, validate(pp, X_train, y_train, groups_train)))

best_max_depth, best_num_leaves, best_score = max(score, key=lambda x: x[2])
params['max_depth'] = best_max_depth
params['num_leaves'] = best_num_leaves
print(f"Best max_depth {params['max_depth']}, num_leaves {params['num_leaves']} with score {best_score}")

Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.387432	training's f1_score: 0.746531	valid_1's binary_logloss: 0.372015	valid_1's f1_score: 0.647887

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[20]	training's binary_logloss: 0.365977	training's f1_score: 0.762564	valid_1's binary_logloss: 0.358946	valid_1's f1_score: 0.659468
[30]	training's binary_logloss: 0.353594	training's f1_score: 0.769835	valid_1's binary_logloss: 0.349828	valid_1's f1_score: 0.664446
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.347858	training's f1_score: 0.774924	valid_1's binary_logloss: 0.348204	valid_1's f1_score: 0.663887
Training until validation scores don't improve for 30 rounds

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[10]	training's binary_logloss: 0.369835	training's f1_score: 0.759633	valid_1's binary_logloss: 0.43147	valid_1's f1_score: 0.589457
[20]	training's binary_logloss: 0.349418	training's f1_score: 0.771521	valid_1's binary_logloss: 0.412838	valid_1's f1_score: 0.616915
[30]	training's binary_logloss: 0.339266	training's f1_score: 0.780052	valid_1's binary_logloss: 0.406159	valid_1's f1_score: 0.616027
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.333052	training's f1_score: 0.782769	valid_1's binary_logloss: 0.399961	valid_1's f1_score: 0.616541

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.381161	training's f1_score: 0.722734	valid_1's binary_logloss: 0.423091	valid_1's f1_score: 0.768828
[20]	training's binary_logloss: 0.357873	training's f1_score: 0.735319	valid_1's binary_logloss: 0.407144	valid_1's f1_score: 0.778892
[30]	training's binary_logloss: 0.347373	training's f1_score: 0.741101	valid_1's binary_logloss: 0.401428	valid_1's f1_score: 0.780591
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.343257	training's f1_score: 0.746354	valid_1's binary_logloss: 0.401888	valid_1's f1_score: 0.784127
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.388515	training's f1_score: 0.733956	valid_1's binary_logloss: 0.360459	valid_1's f1_score: 0.726772

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[20]	training's binary_logloss: 0.367897	training's f1_score: 0.746784	valid_1's binary_logloss: 0.348006	valid_1's f1_score: 0.741355
[30]	training's binary_logloss: 0.356012	training's f1_score: 0.757509	valid_1's binary_logloss: 0.342241	valid_1's f1_score: 0.752407
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.349789	training's f1_score: 0.760132	valid_1's binary_logloss: 0.342017	valid_1's f1_score: 0.75154
Training until validation scores don't improve for 30 rounds

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))


bagging_fractions, feature_fractions = grid_params[("bagging_fraction", "feature_fraction")]
score = []
for bagging_fraction in bagging_fractions:
    for feature_fraction in feature_fractions:
        pp = params.copy()
        pp.update(fixed_params)
        pp["bagging_fraction"] = bagging_fraction
        pp["feature_fraction"] = feature_fraction
        score.append((bagging_fraction, feature_fraction, validate(pp, X_train, y_train, groups_train)))

best_bagging_fraction, best_feature_fraction, best_score = max(score, key=lambda x: x[2])
params['bagging_fraction'] = best_bagging_fraction
params['feature_fraction'] = best_feature_fraction
print(f"Best bagging_fraction {params['bagging_fraction']}, feature_fraction {params['feature_fraction']} with score {best_score}")

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.372477	training's f1_score: 0.763473	valid_1's binary_logloss: 0.378207	valid_1's f1_score: 0.635112
[20]	training's binary_logloss: 0.342811	training's f1_score: 0.776798	valid_1's binary_logloss: 0.366392	valid_1's f1_score: 0.64837
[30]	training's binary_logloss: 0.321414	training's f1_score: 0.789196	valid_1's binary_logloss: 0.364647	valid_1's f1_score: 0.651801
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.307553	training's f1_score: 0.796438	valid_1's binary_logloss: 0.365648	valid_1's f1_score: 0.652632
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.359959	training's f1_score: 0.774497	valid_1's binary_logloss: 0.424055	valid_1's f1_score: 0.605646

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[20]	training's binary_logloss: 0.327529	training's f1_score: 0.784968	valid_1's binary_logloss: 0.42836	valid_1's f1_score: 0.614094
[30]	training's binary_logloss: 0.309158	training's f1_score: 0.800315	valid_1's binary_logloss: 0.431987	valid_1's f1_score: 0.619454
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.29857	training's f1_score: 0.808135	valid_1's binary_logloss: 0.431216	valid_1's f1_score: 0.610871
Training until validation scores don't improve for 30 rounds

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[10]	training's binary_logloss: 0.364796	training's f1_score: 0.738368	valid_1's binary_logloss: 0.427852	valid_1's f1_score: 0.761955
[20]	training's binary_logloss: 0.330913	training's f1_score: 0.763058	valid_1's binary_logloss: 0.426915	valid_1's f1_score: 0.762425
[30]	training's binary_logloss: 0.307952	training's f1_score: 0.778433	valid_1's binary_logloss: 0.429932	valid_1's f1_score: 0.760349
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.298202	training's f1_score: 0.78527	valid_1's binary_logloss: 0.435056	valid_1's f1_score: 0.755556

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.374451	training's f1_score: 0.753455	valid_1's binary_logloss: 0.364795	valid_1's f1_score: 0.731364
[20]	training's binary_logloss: 0.344899	training's f1_score: 0.765759	valid_1's binary_logloss: 0.354755	valid_1's f1_score: 0.733753
[30]	training's binary_logloss: 0.320575	training's f1_score: 0.7796	valid_1's binary_logloss: 0.34951	valid_1's f1_score: 0.735067
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.309602	training's f1_score: 0.787179	valid_1's binary_logloss: 0.351718	valid_1's f1_score: 0.734838
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.370018	training's f1_score: 0.731423	valid_1's binary_logloss: 0.419817	valid_1's f1_score: 0.777668

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[20]	training's binary_logloss: 0.341065	training's f1_score: 0.746274	valid_1's binary_logloss: 0.423916	valid_1's f1_score: 0.77849
[30]	training's binary_logloss: 0.321399	training's f1_score: 0.765087	valid_1's binary_logloss: 0.429736	valid_1's f1_score: 0.780296
Early stopping, best iteration is:
[5]	training's binary_logloss: 0.394772	training's f1_score: 0.718767	valid_1's binary_logloss: 0.435788	valid_1's f1_score: 0.785208
Training until validation scores don't improve for 30 rounds


l1_lambdas, l2_lambdas = grid_params[("lambda_l1", "lambda_l2")]
score = []
for l1_lambda in l1_lambdas:
    for l2_lambda in l2_lambdas:
        pp = params.copy()
        pp.update(fixed_params)
        pp["lambda_l1"] = l1_lambda
        pp["lambda_l2"] = l2_lambda
        score.append((l1_lambda, l2_lambda, validate(pp, X_train, y_train, groups_train)))

best_l1_lambda, best_l2_lambda, best_score = max(score, key=lambda x: x[2])
params['lambda_l1'] = best_l1_lambda
params['lambda_l2'] = best_l2_lambda
print(f"Best lambda_l1 {params['lambda_l1']}, lambda_l2 {params['lambda_l2']} with score {best_score}")

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.329492	training's f1_score: 0.78698	valid_1's binary_logloss: 0.343899	valid_1's f1_score: 0.662196
[20]	training's binary_logloss: 0.291924	training's f1_score: 0.815818	valid_1's binary_logloss: 0.32892	valid_1's f1_score: 0.671875
[30]	training's binary_logloss: 0.261355	training's f1_score: 0.832688	valid_1's binary_logloss: 0.325948	valid_1's f1_score: 0.669611
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.250271	training's f1_score: 0.839944	valid_1's binary_logloss: 0.326469	valid_1's f1_score: 0.676652
Training until validation scores don't improve for 30 rounds

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[10]	training's binary_logloss: 0.312563	training's f1_score: 0.798878	valid_1's binary_logloss: 0.409143	valid_1's f1_score: 0.62
[20]	training's binary_logloss: 0.27999	training's f1_score: 0.815459	valid_1's binary_logloss: 0.401964	valid_1's f1_score: 0.634599
[30]	training's binary_logloss: 0.249242	training's f1_score: 0.836341	valid_1's binary_logloss: 0.401778	valid_1's f1_score: 0.636986
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.234546	training's f1_score: 0.848657	valid_1's binary_logloss: 0.403737	valid_1's f1_score: 0.641026
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.319061	training's f1_score: 0.767776	valid_1's binary_logloss: 0.392159	valid_1's f1_score: 0.778545
[20]	training's binary_logloss: 0.282465	training's f1_score: 0.787858	valid_1's binary_logloss: 0.388385	valid_1's f1_score: 0.787846

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[30]	training's binary_logloss: 0.253262	training's f1_score: 0.815315	valid_1's binary_logloss: 0.385059	valid_1's f1_score: 0.785047
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.238534	training's f1_score: 0.82791	valid_1's binary_logloss: 0.389456	valid_1's f1_score: 0.779736
Training until validation scores don't improve for 30 rounds

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[10]	training's binary_logloss: 0.327167	training's f1_score: 0.779856	valid_1's binary_logloss: 0.34989	valid_1's f1_score: 0.740286
[20]	training's binary_logloss: 0.292254	training's f1_score: 0.79955	valid_1's binary_logloss: 0.345315	valid_1's f1_score: 0.746248
[30]	training's binary_logloss: 0.263794	training's f1_score: 0.822211	valid_1's binary_logloss: 0.343908	valid_1's f1_score: 0.748966
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.247446	training's f1_score: 0.837164	valid_1's binary_logloss: 0.344476	valid_1's f1_score: 0.749307
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.325821	training's f1_score: 0.75489	valid_1's binary_logloss: 0.386559	valid_1's f1_score: 0.793499

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[20]	training's binary_logloss: 0.29266	training's f1_score: 0.777125	valid_1's binary_logloss: 0.398808	valid_1's f1_score: 0.7853
[30]	training's binary_logloss: 0.265842	training's f1_score: 0.798178	valid_1's binary_logloss: 0.400935	valid_1's f1_score: 0.782524
Early stopping, best iteration is:
[4]	training's binary_logloss: 0.364494	training's f1_score: 0.749684	valid_1's binary_logloss: 0.410931	valid_1's f1_score: 0.799195
Training until validation scores don't improve for 30 rounds


thresholds = grid_params['threshold']
score = []
for threshold in tqdm(thresholds):
    pp = params.copy()
    pp.update(fixed_params)
    score.append(validate(pp, X_train, y_train, groups_train, threshold=threshold))
    print(score[-1])

best_threshold = grid_params["threshold"][np.argmax(score)]
max_score = np.max(score)
print(f"Best threshold {best_threshold} with score {max_score}")

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.329492	training's f1_score: 0.78698	valid_1's binary_logloss: 0.343899	valid_1's f1_score: 0.662196
[20]	training's binary_logloss: 0.291924	training's f1_score: 0.815818	valid_1's binary_logloss: 0.32892	valid_1's f1_score: 0.671875
[30]	training's binary_logloss: 0.261355	training's f1_score: 0.832688	valid_1's binary_logloss: 0.325948	valid_1's f1_score: 0.669611
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.250271	training's f1_score: 0.839944	valid_1's binary_logloss: 0.326469	valid_1's f1_score: 0.676652
Training until validation scores don't improve for 30 rounds

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[10]	training's binary_logloss: 0.312563	training's f1_score: 0.798878	valid_1's binary_logloss: 0.409143	valid_1's f1_score: 0.62
[20]	training's binary_logloss: 0.27999	training's f1_score: 0.815459	valid_1's binary_logloss: 0.401964	valid_1's f1_score: 0.634599
[30]	training's binary_logloss: 0.249242	training's f1_score: 0.836341	valid_1's binary_logloss: 0.401778	valid_1's f1_score: 0.636986
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.234546	training's f1_score: 0.848657	valid_1's binary_logloss: 0.403737	valid_1's f1_score: 0.641026
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.319061	training's f1_score: 0.767776	valid_1's binary_logloss: 0.392159	valid_1's f1_score: 0.778545

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[20]	training's binary_logloss: 0.282465	training's f1_score: 0.787858	valid_1's binary_logloss: 0.388385	valid_1's f1_score: 0.787846
[30]	training's binary_logloss: 0.253262	training's f1_score: 0.815315	valid_1's binary_logloss: 0.385059	valid_1's f1_score: 0.785047
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.238534	training's f1_score: 0.82791	valid_1's binary_logloss: 0.389456	valid_1's f1_score: 0.779736

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.327167	training's f1_score: 0.779856	valid_1's binary_logloss: 0.34989	valid_1's f1_score: 0.740286
[20]	training's binary_logloss: 0.292254	training's f1_score: 0.79955	valid_1's binary_logloss: 0.345315	valid_1's f1_score: 0.746248
[30]	training's binary_logloss: 0.263794	training's f1_score: 0.822211	valid_1's binary_logloss: 0.343908	valid_1's f1_score: 0.748966
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.247446	training's f1_score: 0.837164	valid_1's binary_logloss: 0.344476	valid_1's f1_score: 0.749307
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.325821	training's f1_score: 0.75489	valid_1's binary_logloss: 0.386559	valid_1's f1_score: 0.793499

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[20]	training's binary_logloss: 0.29266	training's f1_score: 0.777125	valid_1's binary_logloss: 0.398808	valid_1's f1_score: 0.7853
[30]	training's binary_logloss: 0.265842	training's f1_score: 0.798178	valid_1's binary_logloss: 0.400935	valid_1's f1_score: 0.782524
Early stopping, best iteration is:
[4]	training's binary_logloss: 0.364494	training's f1_score: 0.749684	valid_1's binary_logloss: 0.410931	valid_1's f1_score: 0.799195
0.7043498231162431
Training until validation scores don't improve for 30 rounds


params

{'learning_rate': 0.2653795918367347,
 'max_depth': 5,
 'num_leaves': 22,
 'bagging_fraction': 0.7777777777777778,
 'feature_fraction': 1.0,
 'lambda_l1': 0,
 'lambda_l2': 0}


threshold = 0.59
params = {'learning_rate': 0.2,
          'max_depth': 4,
          'num_leaves': 20,
          'bagging_fraction': 0.8,
          'feature_fraction': 1.0,
          'lambda_l1': 0.01,
          'lambda_l2': 110}
params.update(fixed_params)
validate(params, X_train, y_train, groups_train, threshold)

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.37674	training's f1_score: 0.767617	valid_1's binary_logloss: 0.368802	valid_1's f1_score: 0.664354
[20]	training's binary_logloss: 0.347969	training's f1_score: 0.78113	valid_1's binary_logloss: 0.351762	valid_1's f1_score: 0.681742
[30]	training's binary_logloss: 0.333107	training's f1_score: 0.788308	valid_1's binary_logloss: 0.338153	valid_1's f1_score: 0.690817
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.327166	training's f1_score: 0.791006	valid_1's binary_logloss: 0.334368	valid_1's f1_score: 0.693019
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.358468	training's f1_score: 0.777849	valid_1's binary_logloss: 0.42589	valid_1's f1_score: 0.598347

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[20]	training's binary_logloss: 0.328061	training's f1_score: 0.794892	valid_1's binary_logloss: 0.414307	valid_1's f1_score: 0.619835
[30]	training's binary_logloss: 0.315331	training's f1_score: 0.803616	valid_1's binary_logloss: 0.412389	valid_1's f1_score: 0.628664
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.310195	training's f1_score: 0.805603	valid_1's binary_logloss: 0.412702	valid_1's f1_score: 0.631922

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.365969	training's f1_score: 0.736602	valid_1's binary_logloss: 0.407541	valid_1's f1_score: 0.780142
[20]	training's binary_logloss: 0.339504	training's f1_score: 0.757338	valid_1's binary_logloss: 0.382304	valid_1's f1_score: 0.796583
[30]	training's binary_logloss: 0.328481	training's f1_score: 0.765516	valid_1's binary_logloss: 0.376933	valid_1's f1_score: 0.792716
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.322608	training's f1_score: 0.770411	valid_1's binary_logloss: 0.374192	valid_1's f1_score: 0.795918
Training until validation scores don't improve for 30 rounds

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[10]	training's binary_logloss: 0.376609	training's f1_score: 0.756245	valid_1's binary_logloss: 0.378263	valid_1's f1_score: 0.68012
[20]	training's binary_logloss: 0.34998	training's f1_score: 0.768372	valid_1's binary_logloss: 0.353248	valid_1's f1_score: 0.747875
[30]	training's binary_logloss: 0.335495	training's f1_score: 0.777367	valid_1's binary_logloss: 0.344807	valid_1's f1_score: 0.744847
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.330508	training's f1_score: 0.781546	valid_1's binary_logloss: 0.344208	valid_1's f1_score: 0.738944
Training until validation scores don't improve for 30 rounds
[10]	training's binary_logloss: 0.368314	training's f1_score: 0.730614	valid_1's binary_logloss: 0.399531	valid_1's f1_score: 0.802383

/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_boost_round` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
/home/ilya/miniconda3/envs/sphere-py37/lib/python3.7/site-packages/lightgbm/engine.py:153: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument
  _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))

[20]	training's binary_logloss: 0.344064	training's f1_score: 0.748239	valid_1's binary_logloss: 0.385403	valid_1's f1_score: 0.800588
[30]	training's binary_logloss: 0.326818	training's f1_score: 0.761954	valid_1's binary_logloss: 0.382872	valid_1's f1_score: 0.797637
Did not meet early stopping. Best iteration is:
[36]	training's binary_logloss: 0.320865	training's f1_score: 0.764441	valid_1's binary_logloss: 0.383406	valid_1's f1_score: 0.795868


threshold_xgb = 0.38
param_xgb = {'learning_rate': 0.3,
         'max_depth': 4,
         'max_leaves': 4,
         'subsample': 0.9,
         'colsample_bytree': 1,
         'reg_alpha': 50,
         'reg_lambda': 100,
         'booster': 'gbtree', #gbtree, dart
         'objective': 'binary:logistic',
         'eval_metric': 'logloss'}
threshold_lgbm = 0.59
params_lgbm = {'learning_rate': 0.2,
          'max_depth': 4,
          'num_leaves': 20,
          'bagging_fraction': 0.8,
          'feature_fraction': 1.0,
          'lambda_l1': 0.01,
          'lambda_l2': 110}


def simple_voting(*y_preds):
    def vote(x):
        counts = np.unique(x, return_counts=True)
        return counts[0][np.argmax(counts[1])]
    X = np.column_stack(y_preds)
    return np.apply_along_axis(vote, axis=1, arr=X)

cv = GroupKFold(n_splits=5)
scores = []
for train, val in tqdm(cv.split(X_train, y_train, groups_train)):
    dtrain = xgb.DMatrix(X_train[train], label=y_train[train])
    bst = xgb.train(param, dtrain, 36)
    dval = xgb.DMatrix(X_train[val])
    y_pred_xgb = bst.predict(dval)
    y_pred_xgb[y_pred_xgb >= threshold_xgb] = 1
    y_pred_xgb[y_pred_xgb < threshold_xgb] = 0
    y_pred_xgb = y_pred_xgb.astype(np.int)
    
    train_set = lgb.Dataset(X_train[train], y_train[train])
    val_set = lgb.Dataset(X_train[val], y_train[val])
    model = lgb.train(params_lgbm, train_set, verbose_eval=10,
                      valid_sets=[train_set, val_set], feval=lgbm_f1_score)

    y_pred_lgbm = model.predict(X_train[val])
    y_pred_lgbm = np.where(y_pred_lgbm < threshold_lgbm, 0, 1)
    
    clf_svc = SVC(C=1.025)
    clf_svc.fit(X_train[train], y_train[train])
    y_pred_svc = clf_svc.predict(X_train[val])
    
#     clf_knn = KNeighborsClassifier(n_neighbors=18, n_jobs=-1, weights='distance')
#     clf_knn.fit(X_train[train], y_train[train])
#     y_pred_knn = clf_knn.predict(X_train[val])
    
    mlp_clf1 = MLPClassifier(hidden_layer_sizes=(3, 6), activation='logistic', max_iter=500, random_state=12)
    mlp_clf1.fit(X_train[train], y_train[train])
    y_pred_mlp1 = mlp_clf1.predict(X_train[val])
    
    mlp_clf2 = MLPClassifier(hidden_layer_sizes=(3, 4), activation='relu', max_iter=500, random_state=12)
    mlp_clf2.fit(X_train[train], y_train[train])
    y_pred_mlp2 = mlp_clf2.predict(X_train[val])
    
    y_pred = simple_voting(y_pred_xgb, y_pred_svc, y_pred_mlp1, y_pred_mlp2, y_pred_lgbm)
    scores.append(f1_score(y_true=y_train[val], y_pred=y_pred))
scores

[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000808 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[LightGBM] [Info] Total Bins 15336
[LightGBM] [Info] Number of data points in the train set: 9350, number of used features: 75
[LightGBM] [Info] Start training from score 0.308556
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	training's f1_score: 0.758308	valid_1's f1_score: 0.654545
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	training's f1_score: 0.77705	valid_1's f1_score: 0.672508
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[30]	training's f1_score: 0.787144	valid_1's f1_score: 0.689057
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[40]	training's f1_score: 0.80007	valid_1's f1_score: 0.695187
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[50]	training's f1_score: 0.805886	valid_1's f1_score: 0.694714
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[60]	training's f1_score: 0.810269	valid_1's f1_score: 0.689805
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[70]	training's f1_score: 0.821817	valid_1's f1_score: 0.688312
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[80]	training's f1_score: 0.827102	valid_1's f1_score: 0.681175
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[90]	training's f1_score: 0.831638	valid_1's f1_score: 0.680525
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[100]	training's f1_score: 0.837021	valid_1's f1_score: 0.678337
[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.002226 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 15419
[LightGBM] [Info] Number of data points in the train set: 9347, number of used features: 75
[LightGBM] [Info] Start training from score 0.305767
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	training's f1_score: 0.770218	valid_1's f1_score: 0.573737
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	training's f1_score: 0.79325	valid_1's f1_score: 0.599602
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[30]	training's f1_score: 0.799434	valid_1's f1_score: 0.600976
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[40]	training's f1_score: 0.808631	valid_1's f1_score: 0.599805
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[50]	training's f1_score: 0.813703	valid_1's f1_score: 0.604288
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[60]	training's f1_score: 0.819794	valid_1's f1_score: 0.602151
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[70]	training's f1_score: 0.82426	valid_1's f1_score: 0.600589
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[80]	training's f1_score: 0.828587	valid_1's f1_score: 0.604878
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[90]	training's f1_score: 0.836962	valid_1's f1_score: 0.60878
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[100]	training's f1_score: 0.84289	valid_1's f1_score: 0.610568
[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.002191 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 15417
[LightGBM] [Info] Number of data points in the train set: 9376, number of used features: 75
[LightGBM] [Info] Start training from score 0.268878
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	training's f1_score: 0.717949	valid_1's f1_score: 0.707528
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	training's f1_score: 0.750476	valid_1's f1_score: 0.73927
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[30]	training's f1_score: 0.769392	valid_1's f1_score: 0.746803
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[40]	training's f1_score: 0.784363	valid_1's f1_score: 0.746326
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[50]	training's f1_score: 0.790852	valid_1's f1_score: 0.746479
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[60]	training's f1_score: 0.796589	valid_1's f1_score: 0.744872
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[70]	training's f1_score: 0.802662	valid_1's f1_score: 0.742784
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[80]	training's f1_score: 0.810137	valid_1's f1_score: 0.75
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[90]	training's f1_score: 0.815987	valid_1's f1_score: 0.746803
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[100]	training's f1_score: 0.824089	valid_1's f1_score: 0.74856
[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.003362 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 15412
[LightGBM] [Info] Number of data points in the train set: 9340, number of used features: 75
[LightGBM] [Info] Start training from score 0.293148
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	training's f1_score: 0.746269	valid_1's f1_score: 0.696322
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	training's f1_score: 0.767539	valid_1's f1_score: 0.70598
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[30]	training's f1_score: 0.777085	valid_1's f1_score: 0.717822
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[40]	training's f1_score: 0.788153	valid_1's f1_score: 0.7137
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[50]	training's f1_score: 0.798721	valid_1's f1_score: 0.715221
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[60]	training's f1_score: 0.804896	valid_1's f1_score: 0.713115
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[70]	training's f1_score: 0.810231	valid_1's f1_score: 0.713584
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[80]	training's f1_score: 0.815428	valid_1's f1_score: 0.713584
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[90]	training's f1_score: 0.81974	valid_1's f1_score: 0.712531
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[100]	training's f1_score: 0.830769	valid_1's f1_score: 0.714403
[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.002226 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 15377
[LightGBM] [Info] Number of data points in the train set: 9347, number of used features: 75
[LightGBM] [Info] Start training from score 0.261260
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	training's f1_score: 0.717204	valid_1's f1_score: 0.780343
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	training's f1_score: 0.750433	valid_1's f1_score: 0.79471
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[30]	training's f1_score: 0.767288	valid_1's f1_score: 0.792023
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[40]	training's f1_score: 0.780509	valid_1's f1_score: 0.790304
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[50]	training's f1_score: 0.788228	valid_1's f1_score: 0.791901
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[60]	training's f1_score: 0.79558	valid_1's f1_score: 0.79127
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[70]	training's f1_score: 0.801107	valid_1's f1_score: 0.787437
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[80]	training's f1_score: 0.806376	valid_1's f1_score: 0.78815
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[90]	training's f1_score: 0.813415	valid_1's f1_score: 0.786353
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[100]	training's f1_score: 0.819923	valid_1's f1_score: 0.786555

[0.7082035306334372,
 0.634858812074002,
 0.7632729544034977,
 0.7592295345104335,
 0.7948430493273543]


np.mean(scores)

0.732081576189745


def hard_voting(*y_preds, threshold=0.34):
    X = np.column_stack(y_preds)
    y_pred = X.mean(axis=1)
    y_pred[y_pred >= threshold] = 1
    y_pred[y_pred < threshold] = 0
    y_pred.astype(np.int)
    return y_pred


cv = GroupKFold(n_splits=5)
scores = []
y_preds = []
y_pred_xgb1 = np.zeros(X_test.shape[0])
y_pred_lgbm1 = np.zeros(X_test.shape[0])
y_pred_svc1 = np.zeros(X_test.shape[0])
y_pred_mlp11 = np.zeros(X_test.shape[0])
y_pred_mlp21 = np.zeros(X_test.shape[0])
y_trains = []
for train, val in tqdm(cv.split(X_train, y_train, groups_train)):
    dtrain = xgb.DMatrix(X_train[train], label=y_train[train])
    bst = xgb.train(param, dtrain, 36)
    dval = xgb.DMatrix(X_train[val])
    y_pred_xgb = bst.predict(dval)
    dtest = xgb.DMatrix(X_test)
    y_pred_xgb1 += bst.predict(dtest) / 5
    
    train_set = lgb.Dataset(X_train[train], y_train[train])
    val_set = lgb.Dataset(X_train[val], y_train[val])
    model = lgb.train(params_lgbm, train_set, verbose_eval=10,
                      valid_sets=[train_set, val_set], feval=lgbm_f1_score)
    y_pred_lgbm = model.predict(X_train[val])
    y_pred_lgbm1 += model.predict(X_test) / 5
    
    clf_svc = SVC(C=1.025, probability=True)
    clf_svc.fit(X_train[train], y_train[train])
    y_pred_svc = clf_svc.predict_proba(X_train[val])[:, 1]
    y_pred_svc1 += clf_svc.predict_proba(X_test)[:, 1] /5
    
    mlp_clf1 = MLPClassifier(hidden_layer_sizes=(3, 6), activation='logistic', max_iter=500, random_state=12)
    mlp_clf1.fit(X_train[train], y_train[train])
    y_pred_mlp1 = mlp_clf1.predict_proba(X_train[val])[:, 1]
    y_pred_mlp11 += mlp_clf1.predict_proba(X_test)[:, 1] / 5
    
    mlp_clf2 = MLPClassifier(hidden_layer_sizes=(3, 4), activation='relu', max_iter=500, random_state=12)
    mlp_clf2.fit(X_train[train], y_train[train])
    y_pred_mlp2 = mlp_clf2.predict_proba(X_train[val])[:, 1]
    y_pred_mlp21 += mlp_clf2.predict_proba(X_test)[:, 1] / 5
    
    y_preds.append((y_pred_xgb, y_pred_svc, y_pred_mlp1, y_pred_mlp2, y_pred_lgbm))
    y_trains.append(y_train[val])
    
    y_pred = hard_voting(y_pred_xgb, y_pred_svc, y_pred_mlp1, y_pred_mlp2, y_pred_lgbm)
    scores.append(f1_score(y_true=y_train[val], y_pred=y_pred))
y_pred = hard_voting(y_pred_xgb1, y_pred_svc1, y_pred_mlp11, y_pred_mlp21, y_pred_lgbm1)
scores

[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.002344 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 15135
[LightGBM] [Info] Number of data points in the train set: 9350, number of used features: 69
[LightGBM] [Info] Start training from score 0.308556
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	training's f1_score: 0.748587	valid_1's f1_score: 0.623853
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	training's f1_score: 0.77188	valid_1's f1_score: 0.65867
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[30]	training's f1_score: 0.7777	valid_1's f1_score: 0.660907
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[40]	training's f1_score: 0.781821	valid_1's f1_score: 0.660888
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[50]	training's f1_score: 0.79005	valid_1's f1_score: 0.673118
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[60]	training's f1_score: 0.796935	valid_1's f1_score: 0.673913
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[70]	training's f1_score: 0.804041	valid_1's f1_score: 0.666667
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[80]	training's f1_score: 0.810153	valid_1's f1_score: 0.672414
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[90]	training's f1_score: 0.818624	valid_1's f1_score: 0.680214
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[100]	training's f1_score: 0.823243	valid_1's f1_score: 0.678002
[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.002012 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 15219
[LightGBM] [Info] Number of data points in the train set: 9347, number of used features: 69
[LightGBM] [Info] Start training from score 0.305767
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	training's f1_score: 0.759744	valid_1's f1_score: 0.570546
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	training's f1_score: 0.780505	valid_1's f1_score: 0.594
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[30]	training's f1_score: 0.791425	valid_1's f1_score: 0.59481
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[40]	training's f1_score: 0.80007	valid_1's f1_score: 0.592885
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[50]	training's f1_score: 0.804375	valid_1's f1_score: 0.598612
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[60]	training's f1_score: 0.807977	valid_1's f1_score: 0.597222
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[70]	training's f1_score: 0.815181	valid_1's f1_score: 0.594433
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[80]	training's f1_score: 0.818519	valid_1's f1_score: 0.597815
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[90]	training's f1_score: 0.825577	valid_1's f1_score: 0.593407
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[100]	training's f1_score: 0.830524	valid_1's f1_score: 0.599401
[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.002013 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 15234
[LightGBM] [Info] Number of data points in the train set: 9376, number of used features: 69
[LightGBM] [Info] Start training from score 0.268878
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	training's f1_score: 0.717772	valid_1's f1_score: 0.718937
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	training's f1_score: 0.746678	valid_1's f1_score: 0.747915
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[30]	training's f1_score: 0.760861	valid_1's f1_score: 0.7426
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[40]	training's f1_score: 0.766914	valid_1's f1_score: 0.744423
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[50]	training's f1_score: 0.776378	valid_1's f1_score: 0.748568
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[60]	training's f1_score: 0.786498	valid_1's f1_score: 0.747604
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[70]	training's f1_score: 0.793467	valid_1's f1_score: 0.753181
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[80]	training's f1_score: 0.798597	valid_1's f1_score: 0.750476
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[90]	training's f1_score: 0.807105	valid_1's f1_score: 0.749045
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[100]	training's f1_score: 0.809887	valid_1's f1_score: 0.747604
[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.002447 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 15212
[LightGBM] [Info] Number of data points in the train set: 9340, number of used features: 69
[LightGBM] [Info] Start training from score 0.293148
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	training's f1_score: 0.738834	valid_1's f1_score: 0.719472
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	training's f1_score: 0.762298	valid_1's f1_score: 0.740799
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[30]	training's f1_score: 0.77122	valid_1's f1_score: 0.738654
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[40]	training's f1_score: 0.779699	valid_1's f1_score: 0.738077
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[50]	training's f1_score: 0.787947	valid_1's f1_score: 0.736597
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[60]	training's f1_score: 0.795407	valid_1's f1_score: 0.735938
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[70]	training's f1_score: 0.80068	valid_1's f1_score: 0.732968
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[80]	training's f1_score: 0.804455	valid_1's f1_score: 0.731099
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[90]	training's f1_score: 0.80619	valid_1's f1_score: 0.734789
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[100]	training's f1_score: 0.81172	valid_1's f1_score: 0.735776
[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.001998 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 15181
[LightGBM] [Info] Number of data points in the train set: 9347, number of used features: 69
[LightGBM] [Info] Start training from score 0.261260
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	training's f1_score: 0.710994	valid_1's f1_score: 0.777648
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	training's f1_score: 0.738161	valid_1's f1_score: 0.789146
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[30]	training's f1_score: 0.754175	valid_1's f1_score: 0.786885
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[40]	training's f1_score: 0.765416	valid_1's f1_score: 0.790594
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[50]	training's f1_score: 0.774931	valid_1's f1_score: 0.793296
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[60]	training's f1_score: 0.782498	valid_1's f1_score: 0.795759
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[70]	training's f1_score: 0.78934	valid_1's f1_score: 0.79374
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[80]	training's f1_score: 0.794671	valid_1's f1_score: 0.795289
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[90]	training's f1_score: 0.797712	valid_1's f1_score: 0.792157
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[100]	training's f1_score: 0.804734	valid_1's f1_score: 0.791456

[0.7008086253369272,
 0.6321243523316062,
 0.7863247863247863,
 0.7659574468085106,
 0.7925117004680187]


threshold_grid = np.linspace(0.1, 0.8, 50)

scores_voted = []
for threshold in tqdm(threshold_grid):
    vot_scores = []
    for i, (y_pred_xgb, y_pred_svc, y_pred_mlp1, y_pred_mlp2, y_pred_lgbm) in enumerate(y_preds):
        y_voted = hard_voting(y_pred_xgb, y_pred_mlp1, y_pred_mlp2, y_pred_lgbm, threshold=threshold)
        vot_scores.append(f1_score(y_trains[i], y_voted))
    scores_voted.append(np.mean(vot_scores))
print(f'threshold={threshold_grid[np.argmax(scores_voted)]}\t score={np.max(scores_voted)}')

threshold=0.37142857142857144	 score=0.7354906318960802


y_pred = hard_voting(y_pred_xgb1, y_pred_svc1, y_pred_mlp11, y_pred_mlp21, y_pred_lgbm1, threshold=)
y_pred.shape

(16627,)


cv = GroupKFold(n_splits=5)

y_preds = []

dtrain = xgb.DMatrix(X_train, label=y_train)
bst = xgb.train(param, dtrain, 36)
dtest = xgb.DMatrix(X_test)
y_pred_xgb = bst.predict(dtest)

train_set = lgb.Dataset(X_train, y_train)
model = lgb.train(params_lgbm, train_set, verbose_eval=10)
y_pred_lgbm = model.predict(X_test)

clf_svc = SVC(C=1.025, probability=True)
clf_svc.fit(X_train, y_train)
y_pred_svc = clf_svc.predict_proba(X_test)[:, 1]

mlp_clf1 = MLPClassifier(hidden_layer_sizes=(3, 6), activation='logistic', max_iter=500, random_state=12)
mlp_clf1.fit(X_train, y_train)
y_pred_mlp1 = mlp_clf1.predict_proba(X_test)[:, 1]

mlp_clf2 = MLPClassifier(hidden_layer_sizes=(3, 4), activation='relu', max_iter=500, random_state=12)
mlp_clf2.fit(X_train, y_train)
y_pred_mlp2 = mlp_clf2.predict_proba(X_test)[:, 1]

y_preds.append((y_pred_xgb, y_pred_svc, y_pred_mlp1, y_pred_mlp2, y_pred_lgbm))

y_pred = hard_voting(y_pred_xgb, y_pred_svc, y_pred_mlp1, y_pred_mlp2, y_pred_lgbm)
y_pred.shape

[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.003364 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 15505
[LightGBM] [Info] Number of data points in the train set: 11690, number of used features: 75
[LightGBM] [Info] Start training from score 0.287511
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf

(16627,)


np.unique(y_pred, return_counts=True)

(array([0., 1.]), array([11136,  5491]))


submission = pd.DataFrame({'pair_id': df_test_groups.pair_id, 'target': y_pred.astype(np.int32)})
submission = submission.set_index(['pair_id'])
submission.head()


submission.to_csv('Blending1.csv')


C_grid = np.linspace(0.8, 1.1, 5)
scores = []

for C in C_grid:
    cv = GroupKFold(n_splits)
    clf_svc = SVC(C=C)
    scores.append(np.mean(cross_val_score(clf_svc, X_train, y_train, cv=cv.split(X_train, y_train, groups_train), scoring='f1', n_jobs=-1)))
    print(f'C={C}\tscore={scores[-1]}')

C=0.8	score=0.7174832144875063
C=0.875	score=0.7182516378923312
C=0.9500000000000001	score=0.7194146580971507
C=1.0250000000000001	score=0.7195153743475619
C=1.1	score=0.7191145266911378


C = C_grid[np.argmax(scores)]
C

1.188888888888889


clf_svc = SVC(C=1.025)
cv = GroupKFold(n_splits)
for train, val in cv.split(X_train, y_train, groups_train):
    clf_svc.fit(X_train[train], y_train[train])
    y_pred = clf_svc.predict(X_train[val])
    print(f1_score(y_true=y_train[val], y_pred=y_pred))

0.6913319238900635
0.6167846309403437
0.749519538757207
0.7516447368421052
0.7882960413080895


def calc_rate_matrix(y_pred1, y_pred2, y_true):
    ret = np.zeros((2, 2), dtype=np.int)
    ret[0, 0] = np.sum((y_pred1 == y_pred2) & (y_pred1 == y_true))
    ret[0, 1] = np.sum((y_pred1 == y_true) & (y_pred2 != y_true))
    ret[1, 0] = np.sum((y_pred2 == y_true) & (y_pred1 != y_true))
    ret[1, 1] = np.sum((y_pred1 != y_true) & (y_pred2 != y_true))
    return ret


clf_svc = SVC(C=C)
cv = GroupKFold(n_splits)
for train, val in cv.split(X_train, y_train, groups_train):
    clf_svc.fit(X_train[train], y_train[train])
    dtrain = xgb.DMatrix(X_train[train], label=y_train[train])
    bst = xgb.train(param, dtrain, 43)
    dval = xgb.DMatrix(X_train[val])
    y_xgb = bst.predict(dval)
    threshold = 0.37
    y_xgb[y_xgb >= threshold] = 1
    y_xgb[y_xgb < threshold] = 0
    y_xgb = y_xgb.astype(np.int)
    y_svc = clf_svc.predict(X_train[val])
    print(calc_rate_matrix(y_xgb, y_svc, y_train[val]))

[[1925   89]
 [ 115  211]]
[[1845   98]
 [ 111  289]]
[[1819  117]
 [  91  287]]
[[1923   82]
 [ 106  239]]
[[1860  105]
 [ 101  277]]


clf_svc = SVC(C=C)
clf_logreg = LogisticRegression(C=0.91116276, max_iter=500)
cv = GroupKFold(n_splits)
for train, val in cv.split(X_train, y_train, groups_train):
    clf_svc.fit(X_train[train], y_train[train])
    clf_logreg.fit(X_train[train], y_train[train])
    y_logreg = clf_logreg.predict(X_train[val])
    y_svc = clf_svc.predict(X_train[val])
    print(calc_rate_matrix(y_logreg, y_svc, y_train[val]))

[[1931   80]
 [ 109  220]]
[[1848   60]
 [ 108  327]]
[[1801   70]
 [ 109  334]]
[[1927   62]
 [ 102  259]]
[[1852   82]
 [ 109  300]]


clf_logreg = LogisticRegression(C=0.91116276, max_iter=500)
cv = GroupKFold(n_splits)
for train, val in cv.split(X_train, y_train, groups_train):
    clf_logreg.fit(X_train[train], y_train[train])
    dtrain = xgb.DMatrix(X_train[train], label=y_train[train])
    bst = xgb.train(param, dtrain, 43)
    dval = xgb.DMatrix(X_train[val])
    y_xgb = bst.predict(dval)
    threshold = 0.37
    y_xgb[y_xgb >= threshold] = 1
    y_xgb[y_xgb < threshold] = 0
    y_xgb = y_xgb.astype(np.int)
    y_logreg = clf_logreg.predict(X_train[val])
    print(calc_rate_matrix(y_xgb, y_logreg, y_train[val]))

[[1861  153]
 [ 150  176]]
[[1793  150]
 [ 115  285]]
[[1762  174]
 [ 109  269]]
[[1850  155]
 [ 139  206]]
[[1802  163]
 [ 132  246]]


def simple_voting(*y_preds):
    def vote(x):
        counts = np.unique(x, return_counts=True)
        return counts[0][np.argmax(counts[1])]
    X = np.column_stack(y_preds)
    return np.apply_along_axis(vote, axis=1, arr=X)


clf_logreg = LogisticRegression(C=0.91116276, max_iter=500)
clf_svc = SVC(C=C)
cv = GroupKFold(n_splits)
xgb_scores = []
vot_scores = []
for train, val in cv.split(X_train, y_train, groups_train):
    clf_logreg.fit(X_train[train], y_train[train])
    clf_svc.fit(X_train[train], y_train[train])
    dtrain = xgb.DMatrix(X_train[train], label=y_train[train])
    bst = xgb.train(param, dtrain, 43)
    dval = xgb.DMatrix(X_train[val])
    y_xgb = bst.predict(dval)
    threshold = 0.37
    y_xgb[y_xgb >= threshold] = 1
    y_xgb[y_xgb < threshold] = 0
    y_xgb = y_xgb.astype(np.int)
    y_logreg = clf_logreg.predict(X_train[val])
    y_svc = clf_svc.predict(X_train[val])
    y_voted = simple_voting(y_svc, y_xgb, y_logreg)
    print(calc_rate_matrix(y_xgb, y_voted, y_train[val]))
    print(f'xbg_score: \t{f1_score(y_train[val], y_xgb)}')
    print(f'voted_score: \t{f1_score(y_train[val], y_voted)}')
    xgb_scores.append(f1_score(y_train[val], y_xgb))
    vot_scores.append(f1_score(y_train[val], y_voted))

(2340, 3)
[[1954   60]
 [  99  227]]
xbg_score: 	0.6953271028037384
voted_score: 	0.6890574214517876
(2343, 3)
[[1867   76]
 [  77  323]]
xbg_score: 	0.6428571428571428
voted_score: 	0.5957446808510638
(2314, 3)
[[1849   87]
 [  69  309]]
xbg_score: 	0.7835051546391752
voted_score: 	0.7531172069825437
(2350, 3)
[[1940   65]
 [  94  251]]
xbg_score: 	0.7516198704103672
voted_score: 	0.7443365695792881
(2343, 3)
[[1888   77]
 [  78  300]]
xbg_score: 	0.8033298647242455
voted_score: 	0.787366046249295


def hard_voting(*y_preds, threshold=0.5):
    X = np.column_stack(y_preds)
    y_pred = X.max(axis=1)
    y_pred[y_pred >= threshold] = 1
    y_pred[y_pred < threshold] = 0
    y_pred.astype(np.int)
    return y_pred


#clf_logreg = LogisticRegression(C=0.91116276, max_iter=500)
clf_svc = SVC(C=C, probability=True)
cv = GroupKFold(n_splits)
xgb_scores = []
vot_scores = []
for train, val in cv.split(X_train, y_train, groups_train):
    #clf_logreg.fit(X_train[train], y_train[train])
    clf_svc.fit(X_train[train], y_train[train])
    dtrain = xgb.DMatrix(X_train[train], label=y_train[train])
    bst = xgb.train(param, dtrain, 43)
    dval = xgb.DMatrix(X_train[val])
    y_xgb = bst.predict(dval)
    #y_logreg = clf_logreg.predict_proba(X_train[val])
    y_svc = clf_svc.predict_proba(X_train[val])
    y_voted = hard_voting(y_svc[:, 1], y_xgb, threshold=0.36)
    threshold = 0.37
    y_xgb_ = np.array(y_xgb)
    y_xgb_[y_xgb >= threshold] = 1
    y_xgb_[y_xgb < threshold] = 0
    y_xgb_ = y_xgb_.astype(np.int)
    print(calc_rate_matrix(y_xgb_, y_voted, y_train[val]))
    print(f'xbg_score: \t{f1_score(y_train[val], y_xgb_)}')
    print(f'voted_score: \t{f1_score(y_train[val], y_voted)}')
    xgb_scores.append(f1_score(y_train[val], y_xgb_))
    vot_scores.append(f1_score(y_train[val], y_voted))

[[1960   54]
 [  12  314]]
xbg_score: 	0.6953271028037384
voted_score: 	0.676056338028169
[[1915   28]
 [  16  384]]
xbg_score: 	0.6428571428571428
voted_score: 	0.6460481099656358
[[1927    9]
 [  21  357]]
xbg_score: 	0.7835051546391752
voted_score: 	0.7939189189189189
[[1987   18]
 [   8  337]]
xbg_score: 	0.7516198704103672
voted_score: 	0.7491166077738516
[[1938   27]
 [  26  352]]
xbg_score: 	0.8033298647242455
voted_score: 	0.808101265822785


print(f'xgb mean score: {np.mean(xgb_scores)}')
print(f'voted mean score: {np.mean(vot_scores)}')

xgb mean score: 0.7353278270869339
voted mean score: 0.734648248101872


threshold_grid = np.linspace(0.3, 0.8, 50)

scores_voted = []
for threshold in tqdm(threshold_grid):
    clf_svc = SVC(C=C, probability=True)
    cv = GroupKFold(n_splits)
    xgb_scores = []
    vot_scores = []
    for train, val in cv.split(X_train, y_train, groups_train):
        #clf_logreg.fit(X_train[train], y_train[train])
        clf_svc.fit(X_train[train], y_train[train])
        dtrain = xgb.DMatrix(X_train[train], label=y_train[train])
        bst = xgb.train(param, dtrain, 43)
        dval = xgb.DMatrix(X_train[val])
        y_xgb = bst.predict(dval)
        #y_logreg = clf_logreg.predict_proba(X_train[val])
        y_svc = clf_svc.predict_proba(X_train[val])
        y_voted = hard_voting(y_svc[:, 1], y_xgb, threshold=threshold)
        threshold = 0.37
        y_xgb_ = np.array(y_xgb)
        y_xgb_[y_xgb >= threshold] = 1
        y_xgb_[y_xgb < threshold] = 0
        y_xgb_ = y_xgb_.astype(np.int)
        #print(calc_rate_matrix(y_xgb_, y_voted, y_train[val]))
        #print(f'xbg_score: \t{f1_score(y_train[val], y_xgb_)}')
        #print(f'voted_score: \t{f1_score(y_train[val], y_voted)}')
        xgb_scores.append(f1_score(y_train[val], y_xgb_))
        vot_scores.append(f1_score(y_train[val], y_voted))
    scores_voted.append(np.mean(vot_scores))
print(threshold_grid[np.argmax(scores_voted)], np.max(scores_voted))

0.47346938775510206 0.7383746004368354


scores_voted

[0.730118633850706,
 0.729887040316342,
 0.7311972154390327,
 0.7317506677249945,
 0.732317975626783,
 0.7331307438671264,
 0.7340861403428974,
 0.7353219656674248,
 0.7348493532280624,
 0.7347557438671265,
 0.7360342346824049,
 0.7353138834020101,
 0.7358662254535286,
 0.7366736044210901,
 0.737022540361771,
 0.7377321218198822,
 0.7377825561809297,
 0.7383746004368354,
 0.7372329012275325,
 0.7379754563927937,
 0.7369451299233178,
 0.7372820596566001,
 0.7372008982867218,
 0.7378300172793976,
 0.7369263261569265,
 0.7366262158759079,
 0.736214307403038,
 0.7359366539670154,
 0.7355270443155568,
 0.7352529190083693,
 0.7345271724385549,
 0.7340837070431908,
 0.733330831177953,
 0.7317757203653639,
 0.730353377003221,
 0.7295341869808988,
 0.729033186569177,
 0.7288528312457672,
 0.7271871164161461,
 0.7261411218101251,
 0.7255099912051687,
 0.725731904469385,
 0.723121226100629,
 0.722893674901609,
 0.7222317490551939,
 0.7207452726629379,
 0.719340073216396,
 0.7166561476356325,
 0.7156477164905987,
 0.7127253150981085]


k_grid = np.arange(55, 70, 1)
scores = []
for k in tqdm(k_grid):
    knn_clf = KNeighborsClassifier(n_neighbors=k, n_jobs=-1, weights='distance')
    cv = GroupKFold(n_splits=n_splits)
    cv_gen = cv.split(X_train, y_train, groups_train)
    scores.append(np.mean(cross_val_score(knn_clf, X_train, y_train, groups=groups_train, cv=cv_gen, scoring='f1', n_jobs=-1)))
    print('k={0} score={1}'.format(k, scores[-1]))
print(f'max_score: {np.max(scores)}\t k={k_grid[np.argmax(scores)]}')

k=55 score=0.6862633785164439
k=56 score=0.6863696012027215
k=57 score=0.6863316800114685
k=58 score=0.6854498681916608
k=59 score=0.6841409660884437
k=60 score=0.6878176941718016
k=61 score=0.6869351292437946
k=62 score=0.6874750136756196
k=63 score=0.6860360590833265
k=64 score=0.6866056515744242
k=65 score=0.6858975744909944
k=66 score=0.6853423086433235
k=67 score=0.6864249713552942
k=68 score=0.6868449400378189
k=69 score=0.6857136135812935
max_score: 0.6878176941718016	 k=60


scores = []
# mlp_clf1 = MLPClassifier(hidden_layer_sizes=(5, 3), activation='tanh', max_iter=500, random_state=12)
mlp_clf2 = MLPClassifier(hidden_layer_sizes=(3, 6), activation='logistic', max_iter=500, random_state=12) #4 7
mlp_clf3 = MLPClassifier(hidden_layer_sizes=(3, 4), activation='relu', max_iter=500, random_state=12) #3 2

# cv = GroupKFold(n_splits=n_splits)
# cv_gen = cv.split(X_train, y_train, groups_train)
# score = cross_val_score(mlp_clf1, X_train, y_train, groups=groups_train, cv=cv_gen, scoring='f1', n_jobs=-1)
# print(score)
# scores.append(np.mean(score))
cv = GroupKFold(n_splits=n_splits)
cv_gen = cv.split(X_train, y_train, groups_train)
score = cross_val_score(mlp_clf2, X_train, y_train, groups=groups_train, cv=cv_gen, scoring='f1', n_jobs=-1)
print(score)
scores.append(np.mean(score))
cv = GroupKFold(n_splits=n_splits)
cv_gen = cv.split(X_train, y_train, groups_train)
score = cross_val_score(mlp_clf3, X_train, y_train, groups=groups_train, cv=cv_gen, scoring='f1', n_jobs=-1)
print(score)
scores.append(np.mean(score))
scores

[0.68032787 0.6381323  0.75844806 0.74778405 0.79077615]
[0.68186323 0.6411215  0.77684964 0.7566719  0.78973239]

[0.7230936845506347, 0.7292477308913143]


scores = {'tanh': [],
          'logistic': [],
          'relu': []}
a_grid = np.arange(2, 12)
b_grid = np.arange(2, 12)

for a in tqdm(a_grid):
    for b in b_grid:
        mlp_clf1 = MLPClassifier(hidden_layer_sizes=(a, b), activation='tanh', max_iter=500, random_state=12)
        mlp_clf2 = MLPClassifier(hidden_layer_sizes=(a, b), activation='logistic', max_iter=500, random_state=12)
        mlp_clf3 = MLPClassifier(hidden_layer_sizes=(a, b), activation='relu', max_iter=500, random_state=12)
        
        cv = GroupKFold(n_splits=n_splits)
        cv_gen = cv.split(X_train, y_train, groups_train)
        score = cross_val_score(mlp_clf1, X_train, y_train, groups=groups_train, cv=cv_gen, scoring='f1', n_jobs=-1)
        scores['tanh'].append((a, b, np.mean(score)))
        cv = GroupKFold(n_splits=n_splits)
        cv_gen = cv.split(X_train, y_train, groups_train)
        score = cross_val_score(mlp_clf2, X_train, y_train, groups=groups_train, cv=cv_gen, scoring='f1', n_jobs=-1)
        scores['logistic'].append((a, b, np.mean(score)))
        cv = GroupKFold(n_splits=n_splits)
        cv_gen = cv.split(X_train, y_train, groups_train)
        score = cross_val_score(mlp_clf3, X_train, y_train, groups=groups_train, cv=cv_gen, scoring='f1', n_jobs=-1)
        scores['relu'].append((a, b, np.mean(score)))


scores

{'tanh': [(2, 2, 0.6943962871930374),
  (2, 3, 0.6903965727813665),
  (2, 4, 0.6977752320571178),
  (2, 5, 0.6994285234220813),
  (2, 6, 0.696953036403162),
  (2, 7, 0.6999099876211845),
  (2, 8, 0.69634139481624),
  (2, 9, 0.6961003770518255),
  (2, 10, 0.7002963538932977),
  (2, 11, 0.6880099679293228),
  (3, 2, 0.7036040827277505),
  (3, 3, 0.7034084107982557),
  (3, 4, 0.6954410642640629),
  (3, 5, 0.6886387070863592),
  (3, 6, 0.706621624689636),
  (3, 7, 0.6965633304075943),
  (3, 8, 0.7064025710633073),
  (3, 9, 0.6967540568740473),
  (3, 10, 0.6972282951473148),
  (3, 11, 0.7017790520123665),
  (4, 2, 0.696189202508066),
  (4, 3, 0.7076033137303573),
  (4, 4, 0.704788326204531),
  (4, 5, 0.7036442156208276),
  (4, 6, 0.7022229646924354),
  (4, 7, 0.689926817213115),
  (4, 8, 0.6937433045845537),
  (4, 9, 0.6968161971324152),
  (4, 10, 0.7007018713877395),
  (4, 11, 0.7002761972618126),
  (5, 2, 0.7014195113208161),
  (5, 3, 0.693571125534012),
  (5, 4, 0.6906441235030701),
  (5, 5, 0.7064614601044326),
  (5, 6, 0.7098254948255918),
  (5, 7, 0.6967062947213145),
  (5, 8, 0.690981642016706),
  (5, 9, 0.6924121956257453),
  (5, 10, 0.7070970174428302),
  (5, 11, 0.7018642651468002),
  (6, 2, 0.6948245075393722),
  (6, 3, 0.7013465932648183),
  (6, 4, 0.6998787759597038),
  (6, 5, 0.6941903160139186),
  (6, 6, 0.7016923655182362),
  (6, 7, 0.7017307033643776),
  (6, 8, 0.6974405133670953),
  (6, 9, 0.6996184272773578),
  (6, 10, 0.7002248077458647),
  (6, 11, 0.6965980176032784),
  (7, 2, 0.6988357301792651),
  (7, 3, 0.6978760209551059),
  (7, 4, 0.69972342237836),
  (7, 5, 0.6994446759153563),
  (7, 6, 0.7069976940410573),
  (7, 7, 0.6966198615191449),
  (7, 8, 0.6964093387995957),
  (7, 9, 0.6997712792845796),
  (7, 10, 0.6981246258316595),
  (7, 11, 0.7018864949747671),
  (8, 2, 0.693296031420437),
  (8, 3, 0.6953463538711776),
  (8, 4, 0.705469106178548),
  (8, 5, 0.6899667883514239),
  (8, 6, 0.6988599928076243),
  (8, 7, 0.7045061176411865),
  (8, 8, 0.7003413603966283),
  (8, 9, 0.7099817634690601),
  (8, 10, 0.6959896611477413),
  (8, 11, 0.6967662078929008),
  (9, 2, 0.6962100208213711),
  (9, 3, 0.6996583695223564),
  (9, 4, 0.6983592033899713),
  (9, 5, 0.7072539424814497),
  (9, 6, 0.7069746551266693),
  (9, 7, 0.6999562170474886),
  (9, 8, 0.7034155519086337),
  (9, 9, 0.6936395754288502),
  (9, 10, 0.7049255629013995),
  (9, 11, 0.7046240130175554),
  (10, 2, 0.6956084347115784),
  (10, 3, 0.6932561819766216),
  (10, 4, 0.6965821740273772),
  (10, 5, 0.6936249675119998),
  (10, 6, 0.701984135009446),
  (10, 7, 0.7023078547693766),
  (10, 8, 0.7004070146169127),
  (10, 9, 0.6997490635860173),
  (10, 10, 0.6967284056729722),
  (10, 11, 0.6981550205001487),
  (11, 2, 0.6920881530516576),
  (11, 3, 0.7000960111641413),
  (11, 4, 0.6937280109881514),
  (11, 5, 0.6868775749990499),
  (11, 6, 0.7025329202604738),
  (11, 7, 0.7041903888380011),
  (11, 8, 0.694590030647311),
  (11, 9, 0.6993340577158238),
  (11, 10, 0.6936481292358387),
  (11, 11, 0.697562169700477)],
 'logistic': [(2, 2, 0.7029391894128869),
  (2, 3, 0.7067082564148658),
  (2, 4, 0.692556304182442),
  (2, 5, 0.7003685367852917),
  (2, 6, 0.7000093282510992),
  (2, 7, 0.7020597518161121),
  (2, 8, 0.6970694502119781),
  (2, 9, 0.6932675278522995),
  (2, 10, 0.702076524484279),
  (2, 11, 0.6978242950276905),
  (3, 2, 0.6983199673285088),
  (3, 3, 0.7094527421403336),
  (3, 4, 0.7000426062286806),
  (3, 5, 0.7024809463632071),
  (3, 6, 0.704459857473436),
  (3, 7, 0.7010192602246583),
  (3, 8, 0.6984116096749694),
  (3, 9, 0.7016347050327333),
  (3, 10, 0.7000844625385498),
  (3, 11, 0.7000722585522763),
  (4, 2, 0.7104736713169725),
  (4, 3, 0.7019264842373643),
  (4, 4, 0.7048629548047308),
  (4, 5, 0.7083448801683506),
  (4, 6, 0.7069190699581547),
  (4, 7, 0.7116284435888167),
  (4, 8, 0.704456661240352),
  (4, 9, 0.7015232088152054),
  (4, 10, 0.7021339354443397),
  (4, 11, 0.6938393724246925),
  (5, 2, 0.7053010728371127),
  (5, 3, 0.7087084998284903),
  (5, 4, 0.701115197841261),
  (5, 5, 0.7045812929785489),
  (5, 6, 0.7003644652298588),
  (5, 7, 0.7035083881577285),
  (5, 8, 0.7032810592272338),
  (5, 9, 0.7066571366814728),
  (5, 10, 0.7062802521099769),
  (5, 11, 0.7061368907658),
  (6, 2, 0.7033836734083507),
  (6, 3, 0.7048803746895087),
  (6, 4, 0.7041232893330551),
  (6, 5, 0.7039936034599721),
  (6, 6, 0.6991051673686515),
  (6, 7, 0.6998379865668948),
  (6, 8, 0.6996113661976181),
  (6, 9, 0.7012911909514645),
  (6, 10, 0.6953297282069515),
  (6, 11, 0.6982338876230194),
  (7, 2, 0.7089835780391069),
  (7, 3, 0.701969701739095),
  (7, 4, 0.7064853113272004),
  (7, 5, 0.695668647168681),
  (7, 6, 0.7025349352209701),
  (7, 7, 0.7038787387016491),
  (7, 8, 0.7034106020311957),
  (7, 9, 0.694810600979705),
  (7, 10, 0.7077409787282376),
  (7, 11, 0.7040578606568919),
  (8, 2, 0.709134790578054),
  (8, 3, 0.7096974360883896),
  (8, 4, 0.7022120655959692),
  (8, 5, 0.7057451672870092),
  (8, 6, 0.6970203110313189),
  (8, 7, 0.7078705879965588),
  (8, 8, 0.7055348368274722),
  (8, 9, 0.704579303798852),
  (8, 10, 0.699382286024635),
  (8, 11, 0.702223383840266),
  (9, 2, 0.6965201983006144),
  (9, 3, 0.7046345326644318),
  (9, 4, 0.7063520795704813),
  (9, 5, 0.7038852137760113),
  (9, 6, 0.7021551328549128),
  (9, 7, 0.6964579271478524),
  (9, 8, 0.7056711478808073),
  (9, 9, 0.7049960087609861),
  (9, 10, 0.701118188076324),
  (9, 11, 0.7028643665199239),
  (10, 2, 0.6975896278152506),
  (10, 3, 0.7014987564412547),
  (10, 4, 0.7073096029269665),
  (10, 5, 0.6955540611225539),
  (10, 6, 0.7044553776818926),
  (10, 7, 0.702158942102858),
  (10, 8, 0.6989498843882908),
  (10, 9, 0.7025385992797212),
  (10, 10, 0.7035136896676076),
  (10, 11, 0.7029810395572674),
  (11, 2, 0.6996485919988356),
  (11, 3, 0.7042523380492025),
  (11, 4, 0.7052895658812425),
  (11, 5, 0.7068067287841051),
  (11, 6, 0.6983224451364715),
  (11, 7, 0.6967470734524909),
  (11, 8, 0.7043338626659759),
  (11, 9, 0.7057637447575729),
  (11, 10, 0.6987338393484461),
  (11, 11, 0.7062025742149841)],
 'relu': [(2, 2, 0.6736796012446187),
  (2, 3, 0.7056178969867292),
  (2, 4, 0.675156911564399),
  (2, 5, 0.6916202790546897),
  (2, 6, 0.7072720270085614),
  (2, 7, 0.7068226549920669),
  (2, 8, 0.7047802967093236),
  (2, 9, 0.6769600267562782),
  (2, 10, 0.7077150487527281),
  (2, 11, 0.7107478509858571),
  (3, 2, 0.720866552615297),
  (3, 3, 0.7200753166700349),
  (3, 4, 0.7103756684584662),
  (3, 5, 0.7179543335757994),
  (3, 6, 0.7140822947877302),
  (3, 7, 0.7125065036322071),
  (3, 8, 0.7163134179106316),
  (3, 9, 0.7001728472022707),
  (3, 10, 0.7105818375481279),
  (3, 11, 0.7011379631326449),
  (4, 2, 0.7140044965906626),
  (4, 3, 0.7015437925136377),
  (4, 4, 0.6961449930460561),
  (4, 5, 0.7040310920037681),
  (4, 6, 0.7075427200061168),
  (4, 7, 0.707751193833018),
  (4, 8, 0.70566795040201),
  (4, 9, 0.6957567155368369),
  (4, 10, 0.6927692164024235),
  (4, 11, 0.7145308618662651),
  (5, 2, 0.7025642489084502),
  (5, 3, 0.7063533568251091),
  (5, 4, 0.7116428886421478),
  (5, 5, 0.7040593088299646),
  (5, 6, 0.7139500065991766),
  (5, 7, 0.7102299498372426),
  (5, 8, 0.7024653536714135),
  (5, 9, 0.7007519619553209),
  (5, 10, 0.7001384920925986),
  (5, 11, 0.7042610459463078),
  (6, 2, 0.693583853726105),
  (6, 3, 0.7021922369932637),
  (6, 4, 0.7048469723698035),
  (6, 5, 0.7042856422049233),
  (6, 6, 0.7056142774552747),
  (6, 7, 0.7080624218665517),
  (6, 8, 0.704361975253778),
  (6, 9, 0.7046260807564965),
  (6, 10, 0.7130125450784817),
  (6, 11, 0.71340080568652),
  (7, 2, 0.7000648987248403),
  (7, 3, 0.697540488159275),
  (7, 4, 0.7070575865159585),
  (7, 5, 0.7064726602711022),
  (7, 6, 0.6981173043350235),
  (7, 7, 0.695530625901876),
  (7, 8, 0.6956234371631453),
  (7, 9, 0.6924701861756544),
  (7, 10, 0.7026137662047287),
  (7, 11, 0.6946688269954254),
  (8, 2, 0.7004266854964465),
  (8, 3, 0.7069538227818115),
  (8, 4, 0.7064370718655267),
  (8, 5, 0.6973759148134531),
  (8, 6, 0.7066325705386436),
  (8, 7, 0.6976031510571747),
  (8, 8, 0.7050922776616033),
  (8, 9, 0.698351944013569),
  (8, 10, 0.7016884584378809),
  (8, 11, 0.7056685458806653),
  (9, 2, 0.7041772783168052),
  (9, 3, 0.6980888441031693),
  (9, 4, 0.7048793601505672),
  (9, 5, 0.7037987885677234),
  (9, 6, 0.7021012548175526),
  (9, 7, 0.7000482705395276),
  (9, 8, 0.6933445648392265),
  (9, 9, 0.6957733656965714),
  (9, 10, 0.7014005631613901),
  (9, 11, 0.7037774036953989),
  (10, 2, 0.7025748762641941),
  (10, 3, 0.7047082372678074),
  (10, 4, 0.7086604087194479),
  (10, 5, 0.7055441309256409),
  (10, 6, 0.7086650582505175),
  (10, 7, 0.7050374052434311),
  (10, 8, 0.6970723396209303),
  (10, 9, 0.7037436722225701),
  (10, 10, 0.7021535587968112),
  (10, 11, 0.6931860583821499),
  (11, 2, 0.7016883527494091),
  (11, 3, 0.697669776274653),
  (11, 4, 0.7012214735493155),
  (11, 5, 0.6978622736835263),
  (11, 6, 0.7083058594158439),
  (11, 7, 0.7049640882970989),
  (11, 8, 0.701092844445572),
  (11, 9, 0.6960603144760568),
  (11, 10, 0.701447655088085),
  (11, 11, 0.6980043361354497)]}


for f_activation in scores:
    best_a = scores[f_activation][0][0]
    best_b = scores[f_activation][0][1]
    best_score = scores[f_activation][0][2]
    for a, b, score in scores[f_activation]:
        if score > best_score:
            best_a = a
            best_b = b
            best_score = score
    print(f'score={best_score}\tf_activation=\'{f_activation}\'\ta={best_a}\tb={best_b}')

score=0.7099817634690601	f_activation='tanh'	a=8	b=9
score=0.7116284435888167	f_activation='logistic'	a=4	b=7
score=0.720866552615297	f_activation='relu'	a=3	b=2

	doc_id	title	text
28021	16637	ответ mail ru полезно кушать творог утро худет...	ответ mail ru полезно кушать творог утро худет...
28022	16759	творог полезный свойство лечение творог женски...	творог полезный свойство лечение творог женски...
28023	15358	творог полезный опасный свойство творог	творог полезный опасный свойство творог присое...
28024	17287	ответ mail ru полезный творог	ответ mail ru полезный творог mail ru почта ми...
28025	16026	творог польза вред выбирать продукт jv ru фитн...	творог польза вред выбирать продукт jv ru фитн...

	target
pair_id
11691	1
11692	0
11693	1
11694	1
11695	0

Извлечение данных¶

Извлечение признаков¶

Смотрим базовые модели¶

Логистическая регрессия¶

XGBoost¶

Lightgbm¶

Blending¶

SVC¶

Сравнение базовых моделей и XGBoost'a¶

SVC vs XGBoost¶

LogReg vs SVC¶

LogReg vs XGBoost¶

SimpleVote Prediction¶

MaxProbability prediction¶

KNNClassifier¶

Neural networks¶